<!DOCTYPE html> <html lang=zh-CN style><!--
 Page saved with SingleFile 
 url: https://yuerblog.cc/2020/02/20/k8s-%E5%8E%8B%E6%B5%8B%E5%AF%BC%E8%87%B4%E5%BA%94%E7%94%A8%E9%97%B4%E4%BA%92%E7%9B%B8%E5%BD%B1%E5%93%8D%E7%9A%84%E9%97%AE%E9%A2%98/ 
 saved date: Fri Feb 21 2020 15:27:28 GMT+0800 (China Standard Time)
--><meta charset=utf-8>
<meta name=viewport content="width=device-width">
<title>K8S – 压测导致应用间互相影响的问题 | 鱼儿的博客</title>
<link rel=profile href=http://gmpg.org/xfn/11>
<link rel=pingback href=https://yuerblog.cc/xmlrpc.php>
<!--[if lt IE 9]>
<script src="https://yuerblog.cc/wp-content/themes/twentytwelve/js/html5.js" type="text/javascript"></script>
<![endif]-->
<meta name=description content="这几天对线上K8S集群的应用做性能压测，发现了诡异的现象： 压测A应用，引发了其他几个应用超时报错，然而这些应用间并没有直接或者间接调用关系。">
<meta property=og:locale content=zh_CN>
<meta property=og:type content=website>
<meta property=og:title content="K8S – 压测导致应用间互相影响的问题 | 鱼儿的博客">
<meta property=og:description content="这几天对线上K8S集群的应用做性能压测，发现了诡异的现象： 压测A应用，引发了其他几个应用超时报错，然而这些应用间并没有直接或者间接调用关系。 更诡异的是，某个应用的POD CPU从平日的不到1核瞬间打爆到4核，然而它的流量压根没有因为压测而上升。同时，宿主机load高达数千，但仍旧可以登录宿主机。 注：K8S集群是1.17版本，kube-proxy使用的iptables，应用数量不多，">
<meta property=og:url content=https://yuerblog.cc/2020/02/20/k8s-%e5%8e%8b%e6%b5%8b%e5%af%bc%e8%87%b4%e5%ba%94%e7%94%a8%e9%97%b4%e4%ba%92%e7%9b%b8%e5%bd%b1%e5%93%8d%e7%9a%84%e9%97%ae%e9%a2%98/>
<meta property=og:site_name content=鱼儿的博客>
<meta name=twitter:card content=summary_large_image>
<meta name=twitter:title content="K8S – 压测导致应用间互相影响的问题 | 鱼儿的博客">
<meta name=twitter:description content="这几天对线上K8S集群的应用做性能压测，发现了诡异的现象： 压测A应用，引发了其他几个应用超时报错，然而这些应用间并没有直接或者间接调用关系。 更诡异的是，某个应用的POD CPU从平日的不到1核瞬间打爆到4核，然而它的流量压根没有因为压测而上升。同时，宿主机load高达数千，但仍旧可以登录宿主机。 注：K8S集群是1.17版本，kube-proxy使用的iptables，应用数量不多，">
<link rel=canonical href=https://yuerblog.cc/2020/02/20/k8s-%e5%8e%8b%e6%b5%8b%e5%af%bc%e8%87%b4%e5%ba%94%e7%94%a8%e9%97%b4%e4%ba%92%e7%9b%b8%e5%bd%b1%e5%93%8d%e7%9a%84%e9%97%ae%e9%a2%98/>
<script type=application/ld+json>{"@context":"https://schema.org","@type":"BreadcrumbList","itemListElement":[{"@type":"ListItem","position":1,"item":{"@id":"https://yuerblog.cc/","name":"鱼儿的博客"}},{"@type":"ListItem","position":2,"item":{"@id":"https://yuerblog.cc/category/k8s/","name":"k8s"}},{"@type":"ListItem","position":3,"item":{"@id":"https://yuerblog.cc/2020/02/20/k8s-%e5%8e%8b%e6%b5%8b%e5%af%bc%e8%87%b4%e5%ba%94%e7%94%a8%e9%97%b4%e4%ba%92%e7%9b%b8%e5%bd%b1%e5%93%8d%e7%9a%84%e9%97%ae%e9%a2%98/","name":"K8S &#8211; 压测导致应用间互相影响的问题"}}]}</script>
<link rel=alternate type=application/rss+xml title="鱼儿的博客 » Feed" href=https://yuerblog.cc/feed/>
<link rel=alternate type=application/rss+xml title="鱼儿的博客 » 评论Feed" href=https://yuerblog.cc/comments/feed/>
<link rel=alternate type=application/rss+xml title="鱼儿的博客 » K8S – 压测导致应用间互相影响的问题评论Feed" href=https://yuerblog.cc/2020/02/20/k8s-%e5%8e%8b%e6%b5%8b%e5%af%bc%e8%87%b4%e5%ba%94%e7%94%a8%e9%97%b4%e4%ba%92%e7%9b%b8%e5%bd%b1%e5%93%8d%e7%9a%84%e9%97%ae%e9%a2%98/feed/>
<style>@supports (-webkit-overflow-scrolling:touch){.wp-block-cover-image.has-parallax,.wp-block-cover.has-parallax{background-attachment:scroll}}@supports ((position:-webkit-sticky) or (position:sticky)){.wp-block-cover-image:after,.wp-block-cover:after{content:none}}@supports ((position:-webkit-sticky) or (position:sticky)){.blocks-gallery-grid .blocks-gallery-image figure,.blocks-gallery-grid .blocks-gallery-item figure,.wp-block-gallery .blocks-gallery-image figure,.wp-block-gallery .blocks-gallery-item figure{display:flex;align-items:flex-end;justify-content:flex-start}}@supports ((position:-webkit-sticky) or (position:sticky)){.blocks-gallery-grid .blocks-gallery-image img,.blocks-gallery-grid .blocks-gallery-item img,.wp-block-gallery .blocks-gallery-image img,.wp-block-gallery .blocks-gallery-item img{width:auto}}@supports ((position:-webkit-sticky) or (position:sticky)){.blocks-gallery-grid.is-cropped .blocks-gallery-image a,.blocks-gallery-grid.is-cropped .blocks-gallery-image img,.blocks-gallery-grid.is-cropped .blocks-gallery-item a,.blocks-gallery-grid.is-cropped .blocks-gallery-item img,.wp-block-gallery.is-cropped .blocks-gallery-image a,.wp-block-gallery.is-cropped .blocks-gallery-image img,.wp-block-gallery.is-cropped .blocks-gallery-item a,.wp-block-gallery.is-cropped .blocks-gallery-item img{height:100%;flex:1;-o-object-fit:cover;object-fit:cover}}@media (min-width:600px){@supports (-ms-ime-align:auto){.blocks-gallery-grid.columns-3 .blocks-gallery-image,.blocks-gallery-grid.columns-3 .blocks-gallery-item,.wp-block-gallery.columns-3 .blocks-gallery-image,.wp-block-gallery.columns-3 .blocks-gallery-item{width:calc((100% - 32px)/3 - 1px)}}@supports (-ms-ime-align:auto){.blocks-gallery-grid.columns-4 .blocks-gallery-image,.blocks-gallery-grid.columns-4 .blocks-gallery-item,.wp-block-gallery.columns-4 .blocks-gallery-image,.wp-block-gallery.columns-4 .blocks-gallery-item{width:calc((100% - 48px)/4 - 1px)}}@supports (-ms-ime-align:auto){.blocks-gallery-grid.columns-5 .blocks-gallery-image,.blocks-gallery-grid.columns-5 .blocks-gallery-item,.wp-block-gallery.columns-5 .blocks-gallery-image,.wp-block-gallery.columns-5 .blocks-gallery-item{width:calc((100% - 64px)/5 - 1px)}}@supports (-ms-ime-align:auto){.blocks-gallery-grid.columns-6 .blocks-gallery-image,.blocks-gallery-grid.columns-6 .blocks-gallery-item,.wp-block-gallery.columns-6 .blocks-gallery-image,.wp-block-gallery.columns-6 .blocks-gallery-item{width:calc((100% - 80px)/6 - 1px)}}@supports (-ms-ime-align:auto){.blocks-gallery-grid.columns-7 .blocks-gallery-image,.blocks-gallery-grid.columns-7 .blocks-gallery-item,.wp-block-gallery.columns-7 .blocks-gallery-image,.wp-block-gallery.columns-7 .blocks-gallery-item{width:calc((100% - 96px)/7 - 1px)}}@supports (-ms-ime-align:auto){.blocks-gallery-grid.columns-8 .blocks-gallery-image,.blocks-gallery-grid.columns-8 .blocks-gallery-item,.wp-block-gallery.columns-8 .blocks-gallery-image,.wp-block-gallery.columns-8 .blocks-gallery-item{width:calc((100% - 112px)/8 - 1px)}}}@supports ((-webkit-mask-image:none) or (mask-image:none)) or (-webkit-mask-image:none){.is-style-circle-mask img{-webkit-mask-image:url('data:image/svg+xml;utf8,<svg viewBox="0 0 100 100" xmlns="http://www.w3.org/2000/svg"><circle cx="50" cy="50" r="50"/></svg>');mask-image:url('data:image/svg+xml;utf8,<svg viewBox="0 0 100 100" xmlns="http://www.w3.org/2000/svg"><circle cx="50" cy="50" r="50"/></svg>');mask-mode:alpha;-webkit-mask-repeat:no-repeat;mask-repeat:no-repeat;-webkit-mask-size:contain;mask-size:contain;-webkit-mask-position:center;mask-position:center;border-radius:none}}@supports ((position:-webkit-sticky) or (position:sticky)){.wp-block-video [poster]{-o-object-fit:cover;object-fit:cover}}html,body,div,span,h1,h2,h3,p,blockquote,a,img,small,ul,li,form,label,article,aside,footer,header,hgroup,nav,time{margin:0;padding:0;border:0;font-size:100%;vertical-align:baseline}body{line-height:1}ul{list-style:none}blockquote{quotes:none}blockquote:before,blockquote:after,q:before,q:after{content:'';content:none}h1,h2,h3{clear:both}html{overflow-y:scroll;-webkit-text-size-adjust:100%;-ms-text-size-adjust:100%}a:focus{outline:thin dotted}article,aside,footer,header,hgroup,nav{display:block}small{font-size:smaller}img{border:0;-ms-interpolation-mode:bicubic}.clear:after,.wrapper:after,.format-status .entry-header:after{clear:both}.clear:before,.clear:after,.wrapper:before,.wrapper:after,.format-status .entry-header:before,.format-status .entry-header:after{display:table;content:""}.widget-title{font-size:.785714286rem;line-height:2.181818182;font-weight:bold;text-transform:uppercase;color:#636363}input,textarea{border:1px solid #ccc;border-radius:3px;font-family:inherit;padding:.428571429rem}input{line-height:normal}textarea{overflow:auto;vertical-align:top}input[type="submit"]{padding:.428571429rem .714285714rem;font-size:.785714286rem;line-height:1.428571429;font-weight:normal;color:#7c7c7c;background-color:#e6e6e6;background-repeat:repeat-x;background-image:-moz-linear-gradient(top,#f4f4f4,#e6e6e6);background-image:-ms-linear-gradient(top,#f4f4f4,#e6e6e6);background-image:-webkit-linear-gradient(top,#f4f4f4,#e6e6e6);background-image:-o-linear-gradient(top,#f4f4f4,#e6e6e6);background-image:linear-gradient(to bottom,#f4f4f4,#e6e6e6);border:1px solid #d2d2d2;border-radius:3px;box-shadow:0 1px 2px rgba(64,64,64,0.1)}input[type="submit"]{cursor:pointer}.menu-toggle:hover,.menu-toggle:focus,button:hover,input[type="submit"]:hover,input[type="button"]:hover,input[type="reset"]:hover,article.post-password-required input[type=submit]:hover{color:#5e5e5e;background-color:#ebebeb;background-repeat:repeat-x;background-image:-moz-linear-gradient(top,#f9f9f9,#ebebeb);background-image:-ms-linear-gradient(top,#f9f9f9,#ebebeb);background-image:-webkit-linear-gradient(top,#f9f9f9,#ebebeb);background-image:-o-linear-gradient(top,#f9f9f9,#ebebeb);background-image:linear-gradient(to bottom,#f9f9f9,#ebebeb)}.menu-toggle:active,.menu-toggle.toggled-on,button:active,input[type="submit"]:active,input[type="button"]:active,input[type="reset"]:active{color:#757575;background-color:#e1e1e1;background-repeat:repeat-x;background-image:-moz-linear-gradient(top,#ebebeb,#e1e1e1);background-image:-ms-linear-gradient(top,#ebebeb,#e1e1e1);background-image:-webkit-linear-gradient(top,#ebebeb,#e1e1e1);background-image:-o-linear-gradient(top,#ebebeb,#e1e1e1);background-image:linear-gradient(to bottom,#ebebeb,#e1e1e1);box-shadow:inset 0 0 8px 2px #c6c6c6,0 1px 0 0 #f4f4f4;border-color:transparent}.entry-content img,.widget img{max-width:100%}img[class*="wp-image-"]{height:auto}img.size-full{max-width:100%;height:auto}.aligncenter{display:block;margin-left:auto;margin-right:auto}.entry-content img,.widget img{border-radius:3px;box-shadow:0 1px 4px rgba(0,0,0,0.2)}.site-content nav{clear:both;line-height:2;overflow:hidden}.nav-previous{float:left;width:50%}.nav-next{float:right;text-align:right;width:50%}.nav-single+.comments-area{margin:3.428571429rem 0}html{font-size:87.5%}body{font-size:1rem;font-family:Helvetica,Arial,sans-serif;text-rendering:optimizeLegibility;color:#444}a{outline:0;color:#21759b}a:hover{color:#0f3647}.assistive-text,.site .screen-reader-text{position:absolute!important;clip:rect(1px,1px,1px,1px);overflow:hidden;height:1px;width:1px}.main-navigation .assistive-text:focus,.site .screen-reader-text:hover,.site .screen-reader-text:active,.site .screen-reader-text:focus{background:#fff;border:2px solid #333;border-radius:3px;clip:auto!important;color:#000;display:block;font-size:12px;height:auto;padding:12px;position:absolute;top:5px;left:5px;width:auto;z-index:100000}.site{padding:0 1.714285714rem;background-color:#fff}.site-content{margin:1.714285714rem 0 0}.widget-area{margin:1.714285714rem 0 0}.site-header{padding:1.714285714rem 0}.site-header h1,.site-header h2{text-align:center}.site-header h1 a{color:#515151;display:inline-block;text-decoration:none}.site-header h1 a:hover,.site-header h2 a:hover{color:#21759b}.site-header h1{font-size:1.714285714rem;line-height:1.285714286;margin-bottom:1rem}.site-header h2{font-weight:normal;font-size:.928571429rem;line-height:1.846153846;color:#757575}.main-navigation{margin-top:1.714285714rem;text-align:center}.main-navigation li{margin-top:1.714285714rem;font-size:.857142857rem;line-height:1.42857143}.main-navigation a{color:#5e5e5e}.main-navigation a:hover,.main-navigation a:focus{color:#21759b}.main-navigation ul.nav-menu{display:none}.widget-area .widget{-webkit-hyphens:auto;-moz-hyphens:auto;hyphens:auto;margin-bottom:3.428571429rem;word-wrap:break-word}.widget-area .widget h3{margin-bottom:1.714285714rem}.widget-area .widget p,.widget-area .widget li,.widget-area .widget .textwidget{font-size:.928571429rem;line-height:1.846153846}.widget-area .widget p{margin-bottom:1.714285714rem}.widget-area .widget a{color:#757575}.widget-area .widget a:hover{color:#21759b}.widget-area .widget a:visited{color:#9f9f9f}.widget-area #s{width:53.66666666666%}footer[role="contentinfo"]{border-top:1px solid #ededed;clear:both;font-size:.857142857rem;line-height:2;max-width:68.571428571rem;margin-top:1.714285714rem;margin-left:auto;margin-right:auto;padding:1.714285714rem 0}footer[role="contentinfo"] a{color:#686868}footer[role="contentinfo"] a:hover{color:#21759b}.entry-meta{clear:both}.entry-header{margin-bottom:1.714285714rem}.entry-header .entry-title{font-size:1.428571429rem;line-height:1.2;font-weight:normal}.entry-header .comments-link{margin-top:1.714285714rem;font-size:.928571429rem;line-height:1.846153846;color:#757575}.comments-link a,.entry-meta a{color:#757575}.comments-link a:hover,.entry-meta a:hover{color:#21759b}.entry-content{line-height:1.714285714}.entry-content h1{margin:1.714285714rem 0}.entry-content h1{font-size:1.5rem;line-height:1.5}.entry-content p{margin:0 0 1.714285714rem;line-height:1.714285714}.entry-content a:visited,.comment-content a:visited{color:#9f9f9f}.entry-content ul{margin:0 0 1.714285714rem;line-height:1.714285714}.entry-content ul{list-style:disc outside}.entry-content li{margin:0 0 0 2.571428571rem}.entry-content blockquote{margin-bottom:1.714285714rem;padding:1.714285714rem;font-style:italic}.entry-content blockquote p:last-child,.comment-content blockquote p:last-child{margin-bottom:0}img.aligncenter{clear:both;margin-top:.857142857rem;margin-bottom:.857142857rem}.site-content article{border-bottom:4px double #ededed;margin-bottom:5.142857143rem;padding-bottom:1.714285714rem;word-wrap:break-word;-webkit-hyphens:auto;-moz-hyphens:auto;hyphens:auto}footer.entry-meta{margin-top:1.714285714rem;font-size:.928571429rem;line-height:1.846153846;color:#757575}#respond{margin-top:3.428571429rem}#respond h3#reply-title{font-size:1.142857143rem;line-height:1.5}#respond h3#reply-title #cancel-comment-reply-link{margin-left:.714285714rem;font-weight:normal;font-size:.857142857rem}#respond form{margin:1.714285714rem 0}#respond form p{margin:.785714286rem 0}#respond form label{display:block;line-height:1.714285714}#respond form input[type="text"],#respond form textarea{-moz-box-sizing:border-box;box-sizing:border-box;font-size:.857142857rem;line-height:1.714285714;padding:.714285714rem;width:100%}.widget_search label{display:block;font-size:.928571429rem;line-height:1.846153846}@-ms-viewport{width:device-width}@viewport{width:device-width}@media screen and (min-width:600px){.site{margin:0 auto;max-width:68.571428571rem;overflow:hidden}.site-content{float:left;width:65.104166667%}.widget-area{float:right;width:26.041666667%}.site-header h1,.site-header h2{text-align:left}.site-header h1{font-size:1.857142857rem;line-height:1.846153846;margin-bottom:0}.main-navigation ul.nav-menu{border-bottom:1px solid #ededed;border-top:1px solid #ededed;display:inline-block!important;text-align:left;width:100%}.main-navigation ul{margin:0;text-indent:0}.main-navigation li a,.main-navigation li{display:inline-block;text-decoration:none}.main-navigation li a{border-bottom:0;color:#6a6a6a;line-height:3.692307692;text-transform:uppercase;white-space:nowrap}.main-navigation li a:hover,.main-navigation li a:focus{color:#000}.main-navigation li{margin:0 2.857142857rem 0 0;position:relative}.entry-header .entry-title{font-size:1.571428571rem}#respond form input[type="text"]{width:46.333333333%}}@media screen and (min-width:960px){body{background-color:#e6e6e6}body .site{padding:0 2.857142857rem;margin-top:3.428571429rem;margin-bottom:3.428571429rem;box-shadow:0 2px 6px rgba(100,100,100,0.3)}}</style>
<!--[if lt IE 9]>
<link rel='stylesheet' id='twentytwelve-ie-css'  href='https://yuerblog.cc/wp-content/themes/twentytwelve/css/ie.css?ver=20121010' type='text/css' media='all' />
<![endif]-->
<link rel=https://api.w.org/ href=https://yuerblog.cc/wp-json/>
<link rel=EditURI type=application/rsd+xml title=RSD href=https://yuerblog.cc/xmlrpc.php?rsd>
<link rel=wlwmanifest type=application/wlwmanifest+xml href=https://yuerblog.cc/wp-includes/wlwmanifest.xml>
<link rel=alternate type=application/json+oembed href="https://yuerblog.cc/wp-json/oembed/1.0/embed?url=https%3A%2F%2Fyuerblog.cc%2F2020%2F02%2F20%2Fk8s-%25e5%258e%258b%25e6%25b5%258b%25e5%25af%25bc%25e8%2587%25b4%25e5%25ba%2594%25e7%2594%25a8%25e9%2597%25b4%25e4%25ba%2592%25e7%259b%25b8%25e5%25bd%25b1%25e5%2593%258d%25e7%259a%2584%25e9%2597%25ae%25e9%25a2%2598%2F">
<link rel=alternate type=text/xml+oembed href="https://yuerblog.cc/wp-json/oembed/1.0/embed?url=https%3A%2F%2Fyuerblog.cc%2F2020%2F02%2F20%2Fk8s-%25e5%258e%258b%25e6%25b5%258b%25e5%25af%25bc%25e8%2587%25b4%25e5%25ba%2594%25e7%2594%25a8%25e9%2597%25b4%25e4%25ba%2592%25e7%259b%25b8%25e5%25bd%25b1%25e5%2593%258d%25e7%259a%2584%25e9%2597%25ae%25e9%25a2%2598%2F&amp;format=xml">
<style id=custom-background-css>body.custom-background{background-image:url();background-position:center center;background-size:auto;background-repeat:repeat;background-attachment:fixed}</style>
<link type=image/x-icon rel="shortcut icon" href=data:><style>.sf-hidden{display:none!important}</style></head>
<body class="post-template-default single single-post postid-5747 single-format-standard custom-background wp-embed-responsive single-author">
<div id=page class="hfeed site">
 <header id=masthead class=site-header role=banner>
 <hgroup>
 <h1 class=site-title><a href=https://yuerblog.cc/ title=鱼儿的博客 rel=home class>鱼儿的博客</a></h1>
 <h2 class=site-description>但行好事，莫问前程</h2>
 </hgroup>
 <nav id=site-navigation class=main-navigation role=navigation>
 <button class="menu-toggle sf-hidden">菜单</button>
 <a class=assistive-text href=#content title=跳至正文>跳至正文</a>
 <div class=menu-%e9%a1%b6%e9%83%a8%e8%8f%9c%e5%8d%95-container><ul id=menu-%e9%a1%b6%e9%83%a8%e8%8f%9c%e5%8d%95 class=nav-menu><li id=menu-item-3204 class="menu-item menu-item-type-custom menu-item-object-custom menu-item-home menu-item-3204"><a href=http://yuerblog.cc/>首页</a></li>
<li id=menu-item-279 class="menu-item menu-item-type-post_type menu-item-object-post menu-item-279"><a href=https://yuerblog.cc/2016/09/01/recommended-articles/ class>好文推荐</a></li>
<li id=menu-item-5634 class="menu-item menu-item-type-post_type menu-item-object-post menu-item-5634"><a href=https://yuerblog.cc/2020/01/10/b%e7%ab%99%e7%9b%b4%e6%92%ad%e5%9b%9e%e6%94%be-%e7%bd%ae%e9%a1%b6/><font color=#fb7299 class>B站视频</font></a></li>
<li id=menu-item-1189 class="menu-item menu-item-type-post_type menu-item-object-post menu-item-1189"><a href=https://yuerblog.cc/2017/02/15/booklist/>读书清单</a></li>
<li id=menu-item-1787 class="menu-item menu-item-type-post_type menu-item-object-post menu-item-1787"><a href=https://yuerblog.cc/2017/06/18/care-about-health/ class>关注健康</a></li>
<li id=menu-item-3266 class="menu-item menu-item-type-post_type menu-item-object-post menu-item-3266"><a href=https://yuerblog.cc/2018/06/02/about-my-blog/ class>关于博客</a></li>
</ul></div> </nav>
 </header>
 <div id=main class=wrapper>
 <div id=primary class=site-content>
 <div id=content role=main class>
 
 
 <article id=post-5747 class="post-5747 post type-post status-publish format-standard hentry category-k8s">
 <header class=entry-header>
 
 <h1 class=entry-title>K8S – 压测导致应用间互相影响的问题</h1>
 <div class=comments-link>
 <a href=https://yuerblog.cc/2020/02/20/k8s-%e5%8e%8b%e6%b5%8b%e5%af%bc%e8%87%b4%e5%ba%94%e7%94%a8%e9%97%b4%e4%ba%92%e7%9b%b8%e5%bd%b1%e5%93%8d%e7%9a%84%e9%97%ae%e9%a2%98/#respond><span class=leave-reply>发表评论</span></a> </div>
 </header>
 <div class=entry-content>
 <div class=yuerb-before-content_2 id=yuerb-1534903892>
<ins class="adsbygoogle sf-hidden" style=display:block;text-align:center data-ad-layout=in-article data-ad-format=fluid data-ad-client=ca-pub-5878478910106641 data-ad-slot=5599238126></ins>
</div><p class>这几天对线上K8S集群的应用做性能压测，发现了诡异的现象：<span id=more-5747></span></p>
<blockquote><p>压测A应用，引发了其他几个应用超时报错，然而这些应用间并没有直接或者间接调用关系。</p></blockquote>
<p>更诡异的是，某个应用的POD CPU从平日的不到1核瞬间打爆到4核，然而它的流量压根没有因为压测而上升。同时，宿主机load高达数千，但仍旧可以登录宿主机。</p>
<p>注：K8S集群是1.17版本，kube-proxy使用的iptables，应用数量不多，服务对外暴露采用云厂商的LoadBalancer Service（cluster模式）。</p>
<h1>复现</h1>
<p>经过天马行空的排查，最终决定逐渐增加A的压测流量，看问题什么时候复现，以便观察现场。</p>
<p>为了排除各种嫌疑，我们把A应用亲和到了独立的Node上独占资源。</p>
<p>通过逐步增加压测流量，当到达500Mbps的时候发现问题复现，也就是说其他node上的应用又出现了响应超时。</p>
<p>此时A应用的redis负载已经被打满，A应用连接redis开始报错重试，为了避免该嫌疑，我们扩容了1台redis从库，A应用不再报错，同时其他应用也没有了超时问题。</p>
<h1>原因</h1>
<p>虽然POD已经隔离到独立的node，但是其他应用的LoadBalancer使用的cluster模式，还是会将流量打到A应用所在的node。</p>
<p>当A应用redis超载后，A应用开始大量重连redis，导致短链接突增，其node上的conntrack表记录数打高，进而影响了对其他应用service流量的iptables转发性能，导致其他应用被访问超时。</p>
<h1>关于conntrack</h1>
<p>k8s的service原理就是基于iptables做DNAT转发（kube-proxy也可以选择ipvs模式），转发关系会维护在内核conntrack哈希表中。</p>
<p>当因为重连导致大量短链接建立时，conntrack表中的记录数增多，因为没有对conntrack哈希表进行过优化，所以哈希表的拉链变长，查找效率降低。</p>
<p>实际上iptables的性能需要从2方面看待：</p>
<ul>
<li>一个是当有大量service在K8S集群内的时候，iptables规则匹配的耗时会增加。</li>
<li>一个是当node有大量连接数的时候，conntrack查表的耗时会增加。</li>
</ul>
<p>大家应该深入了解iptables原理与使用，才能理解conntrack表存了什么以及如何工作的，大家可以在我的其他博客中找到相关资料。</p>
<p>关于conntrack哈希表的优化，只需要把哈希桶的数量调大即可，大家可以参考《<a href=https://www.starduster.me/2019/07/05/nf-conntrack-tuning/>nf_conntrack 的数据结构和相关参数调优</a>》。</p>
<p>简单的说，大家需要编辑/etc/sysctl.conf文件，配置conntrack如下参数：</p>
<ul>
<li>net.netfilter.nf_conntrack_buckets：哈希桶的数量，越多则性能越好，冲突率越低，桶内拉链越短。</li>
<li>net.netfilter.nf_conntrack_max：可以容纳的连接总数，利用net.netfilter.nf_conntrack_max / net.netfilter.nf_conntrack_buckets可以得到每个桶内拉链最多可以容纳多少个连接，我们要确保拉链长度尽量保持在个位数、十位数、百位数这种量级，否则遍历查找链表将导致性能下降。</li>
</ul>
<h1>关于LoadBalancer</h1>
<p>我们使用的是LoadBalancer类型的service，大部分云厂商的实现都是首先让service作为nodeport暴露端口，然后在K8S集群外创建云平台的LB负载均衡，并把LB的IP填写到service的status里。</p>
<p>在K8S官网中，LoadBalancer的service支持2种转发模式，可以在service的YAML中配置：</p>
<ul>
<li>externalTrafficPolicy：local</li>
<li>externalTrafficPolicy：cluster</li>
</ul>
<p>cluster模式是K8S集群外LB会向所有K8S node转发流量，并且LB会做SNAT再发给node，同时node收到流量后会随机选择一个POD再做DNAT转发，同时也会SNAT再次修改源地址。</p>
<p>cluster模式的缺点是：</p>
<ul>
<li>流量先随机打到某个node，再DNAT二次转发给随机的POD（可能在其他Node上），导致集群内流量翻倍。</li>
<li>经过LB和转发node的SNAT后，客户端真实的源IP地址已经不见了。</li>
</ul>
<p>我们压测过程中即便把A应用隔离在独立Node，也依然会影响其他应用的原因就是因为cluster模式。</p>
<p>而使用local模式可以保留客户端真实IP，因此：</p>
<ul>
<li>LB是不会做SNAT的，因此POD应答时会直接发回给客户端而不经过LB。</li>
<li>node上只会维护本机POD的iptables规则，因此不会向其他node上的POD做DNAT转发，因此也就不会做SNAT变更源地址。</li>
</ul>
<p>基于上述2点，可以实现保留请求源IP的目的，但是问题也显而易见：</p>
<blockquote class><p>LB必须知道Service的POD在哪些node上，避免向没有该service POD的node转发流量。</p>
<p class>LB要实现这个的方法大部分就是探测所有node上该service的nodeport是否可以联通，因为local模式的iptables规则不会做2次转发，如果不存在本机pod则连接会立即失败。</p></blockquote>
<p class>所以，如果POD发生了node迁移，而LB探测不及时的话，可能造成流量仍旧发往没有该service POD的node，请求就会失败。</p>
<p>这里有一个案例分享给大家：<a href=https://www.bookstack.cn/read/kubernetes-practice-guide/troubleshooting-cases-lb-with-local-externaltrafficpolicy-timeout-occasionally.md>《访问 externalTrafficPolicy 为 Local 的 Service 对应 LB 有时超时》</a></p>
<blockquote class><p class>根据云厂商提供的经验值，单机跑满1000Mbps应该是没有问题的，前提是conntrack表做好优化。</p></blockquote>
 </div>
 
 <footer class=entry-meta>
 本条目发布于<a href=https://yuerblog.cc/2020/02/20/k8s-%e5%8e%8b%e6%b5%8b%e5%af%bc%e8%87%b4%e5%ba%94%e7%94%a8%e9%97%b4%e4%ba%92%e7%9b%b8%e5%bd%b1%e5%93%8d%e7%9a%84%e9%97%ae%e9%a2%98/ title=下午4:24 rel=bookmark><time class=entry-date datetime=2020-02-20T16:24:25+08:00>2020年2月20日</time></a>。属于<a href=https://yuerblog.cc/category/k8s/ rel="category tag" class>k8s</a>分类。<span class="by-author sf-hidden">作者是<span class="author vcard"></span>。</span> </footer>
 </article>
 
 
<div id=comments class=comments-area>
 
 
 
 
</div>
 
 </div>
 </div>
 
 </div>
 
</div>
<script data-template-shadow-root>(() => { document.currentScript.remove(); const processNode = node => { node.querySelectorAll("template[shadowmode]").forEach(element=>{ if (!element.parentElement.shadowRoot) { const shadowRoot = element.parentElement.attachShadow({mode:element.getAttribute("shadowmode"),delegatesFocus:Boolean(element.getAttribute("delegatesfocus"))}); shadowRoot.innerHTML = element.innerHTML; element.remove(); processNode(shadowRoot); } }) }; const FORBIDDEN_TAG_NAMES = ["a","area","audio","base","br","col","command","embed","hr","img","iframe","input","keygen","link","meta","param","source","track","video","wbr"]; const NOTE_TAGNAME = "single-file-note"; const NOTE_CLASS = "note"; const NOTE_ANCHORED_CLASS = "note-anchored"; const NOTE_SELECTED_CLASS = "note-selected"; const NOTE_MOVING_CLASS = "note-moving"; const NOTE_MASK_MOVING_CLASS = "note-mask-moving"; const MASK_CLASS = "single-file-mask"; const HIGHLIGHT_CLASS = "single-file-highlight"; const NOTES_WEB_STYLESHEET = ".note { all: initial; display: flex; flex-direction: column; height: 150px; width: 150px; position: absolute; top: 10px; left: 10px; border: 1px solid rgb(191, 191, 191); z-index: 2147483646; box-shadow: 2px 2px 5px rgba(33, 33, 33, .7); min-height: 100px; min-width: 100px; } .note-selected { z-index: 2147483647; } .note-hidden { display: none; } .note-collapsed { min-height: 30px; max-height: 30px; overflow: hidden; } .note textarea { all: initial; white-space: pre; font-family: Arial, Helvetica, sans-serif; font-size: 14px; padding: 3px; height: 100%; border: 1px solid transparent; resize: none; } .note textarea:focus { border: 1px dotted rgb(160, 160, 160); } .note header { all: initial; min-height: 30px; cursor: grab; user-select: none; } .note .note-remove { all: initial; position: absolute; right: 0px; top: 2px; padding: 5px; opacity: .5; cursor: pointer; user-select: none; } .note .note-anchor { all: initial; position: absolute; left: 0px; top: 2px; padding: 5px; opacity: .25; cursor: pointer; } .note .note-resize { all: initial; position: absolute; bottom: -5px; right: -5px; height: 15px; width: 15px; cursor: nwse-resize; user-select: none; } .note .note-remove:hover { opacity: 1; } .note .note-anchor:hover { opacity: .5; } .note-anchored .note-anchor { opacity: .5; } .note-anchored .note-anchor:hover { opacity: 1; } .note-moving { opacity: .5; } .note-moving * { cursor: grabbing; } .note-yellow header { background-color: #f5f545; } .note-yellow textarea { background-color: #ffff7c; } .note-pink header { background-color: #ffa59f; } .note-pink textarea { background-color: #ffbbb6; } .note-blue header { background-color: #84c8ff; } .note-blue textarea { background-color: #95d0ff; } .note-green header { background-color: #93ef8d; } .note-green textarea { background-color: #9cff95; }"; const MASK_WEB_STYLESHEET = ".note-mask { all: initial; position: fixed; z-index: 2147483645; pointer-events: none; } .note-mask-moving { background-color: rgba(236, 234, 234, 0.5); } .page-mask { all: initial; position: fixed; top: 0; left: 0; width: 0; height: 0; z-index: 2147483646; } .page-mask-active { width: 100vw; height: 100vh; }"; const NOTE_HEADER_HEIGHT = 25; const PAGE_MASK_ACTIVE_CLASS = "page-mask-active"; const REMOVED_CONTENT_CLASS = "single-file-removed"; const reflowNotes = function reflowNotes() { document.querySelectorAll(NOTE_TAGNAME).forEach(containerElement => { const noteElement = containerElement.shadowRoot.querySelector("." + NOTE_CLASS); const noteBoundingRect = noteElement.getBoundingClientRect(); const anchorElement = getAnchorElement(containerElement); const anchorBoundingRect = anchorElement.getBoundingClientRect(); const maxX = anchorBoundingRect.x + Math.max(0, anchorBoundingRect.width - noteBoundingRect.width); const minX = anchorBoundingRect.x; const maxY = anchorBoundingRect.y + Math.max(0, anchorBoundingRect.height - NOTE_HEADER_HEIGHT); const minY = anchorBoundingRect.y; let left = parseInt(noteElement.style.getPropertyValue("left")); let top = parseInt(noteElement.style.getPropertyValue("top")); if (noteBoundingRect.x > maxX) { left -= noteBoundingRect.x - maxX; } if (noteBoundingRect.x < minX) { left += minX - noteBoundingRect.x; } if (noteBoundingRect.y > maxY) { top -= noteBoundingRect.y - maxY; } if (noteBoundingRect.y < minY) { top += minY - noteBoundingRect.y; } noteElement.style.setProperty("position", "absolute"); noteElement.style.setProperty("left", left + "px"); noteElement.style.setProperty("top", top + "px"); }); }; const addNoteRef = function addNoteRef(anchorElement, noteId) { const noteRefs = getNoteRefs(anchorElement); noteRefs.push(noteId); setNoteRefs(anchorElement, noteRefs); }; const deleteNoteRef = function deleteNoteRef(containerElement, noteId) { const anchorElement = getAnchorElement(containerElement); const noteRefs = getNoteRefs(anchorElement).filter(noteRefs => noteRefs != noteId); if (noteRefs.length) { setNoteRefs(anchorElement, noteRefs); } else { delete anchorElement.dataset.singleFileNoteRefs; } }; const getNoteRefs = function getNoteRefs(anchorElement) { return JSON.parse("[" + (anchorElement.dataset.singleFileNoteRefs || "") + "]"); }; const setNoteRefs = function setNoteRefs(anchorElement, noteRefs) { anchorElement.dataset.singleFileNoteRefs = noteRefs.toString(); }; const getAnchorElement = function getAnchorElement(containerElement) { return document.querySelector("[data-single-file-note-refs^=" + JSON.stringify(containerElement.dataset.noteId) + "], [data-single-file-note-refs$=" + JSON.stringify(containerElement.dataset.noteId) + "], [data-single-file-note-refs*=" + JSON.stringify("," + containerElement.dataset.noteId + ",") + "]"); }; const getMaskElement = function getMaskElement(className, containerClassName) { let maskElement = document.documentElement.querySelector("." + className); if (!maskElement) { maskElement = document.createElement("div"); const maskContainerElement = document.createElement("div"); if (containerClassName) { maskContainerElement.classList.add(containerClassName); } maskContainerElement.classList.add(MASK_CLASS); const firstNote = document.querySelector(NOTE_TAGNAME); if (firstNote && firstNote.parentElement == document.documentElement) { document.documentElement.insertBefore(maskContainerElement, firstNote); } else { document.documentElement.appendChild(maskContainerElement); } maskElement.classList.add(className); const maskShadow = maskContainerElement.attachShadow({ mode: "open" }); maskShadow.appendChild(getStyleElement(MASK_WEB_STYLESHEET)); maskShadow.appendChild(maskElement); return maskElement; } }; const getStyleElement = function getStyleElement(stylesheet) { const linkElement = document.createElement("style"); linkElement.textContent = stylesheet; return linkElement; }; const attachNoteListeners = function attachNoteListeners(containerElement, editable = false) { const SELECT_PX_THRESHOLD = 4; const COLLAPSING_NOTE_DELAY = 750; const noteShadow = containerElement.shadowRoot; const noteElement = noteShadow.childNodes[1]; const headerElement = noteShadow.querySelector("header"); const mainElement = noteShadow.querySelector("textarea"); const noteId = containerElement.dataset.noteId; const resizeElement = noteShadow.querySelector(".note-resize"); const anchorIconElement = noteShadow.querySelector(".note-anchor"); const removeNoteElement = noteShadow.querySelector(".note-remove"); mainElement.readOnly = !editable; if (!editable) { anchorIconElement.style.setProperty("display", "none", "important"); } else { anchorIconElement.style.removeProperty("display"); } headerElement.ontouchstart = headerElement.onmousedown = event => { if (event.target == headerElement) { collapseNoteTimeout = setTimeout(() => noteElement.classList.toggle("note-collapsed"), COLLAPSING_NOTE_DELAY); event.preventDefault(); const position = getPosition(event); const clientX = position.clientX; const clientY = position.clientY; const boundingRect = noteElement.getBoundingClientRect(); const deltaX = clientX - boundingRect.left; const deltaY = clientY - boundingRect.top; maskPageElement.classList.add(PAGE_MASK_ACTIVE_CLASS); document.documentElement.style.setProperty("user-select", "none", "important"); anchorElement = getAnchorElement(containerElement); displayMaskNote(); selectNote(noteElement); moveNote(event, deltaX, deltaY); movingNoteMode = { event, deltaX, deltaY }; document.documentElement.ontouchmove = document.documentElement.onmousemove = event => { clearTimeout(collapseNoteTimeout); if (!movingNoteMode) { movingNoteMode = { deltaX, deltaY }; } movingNoteMode.event = event; moveNote(event, deltaX, deltaY); }; } }; resizeElement.ontouchstart = resizeElement.onmousedown = event => { event.preventDefault(); resizingNoteMode = true; selectNote(noteElement); maskPageElement.classList.add(PAGE_MASK_ACTIVE_CLASS); document.documentElement.style.setProperty("user-select", "none", "important"); document.documentElement.ontouchmove = document.documentElement.onmousemove = event => { event.preventDefault(); const { clientX, clientY } = getPosition(event); const boundingRectNote = noteElement.getBoundingClientRect(); noteElement.style.width = clientX - boundingRectNote.left + "px"; noteElement.style.height = clientY - boundingRectNote.top + "px"; }; }; anchorIconElement.ontouchend = anchorIconElement.onclick = event => { event.preventDefault(); noteElement.classList.toggle(NOTE_ANCHORED_CLASS); if (!noteElement.classList.contains(NOTE_ANCHORED_CLASS)) { deleteNoteRef(containerElement, noteId); addNoteRef(document.documentElement, noteId); } }; removeNoteElement.ontouchend = removeNoteElement.onclick = event => { event.preventDefault(); deleteNoteRef(containerElement, noteId); containerElement.remove(); }; noteElement.onmousedown = () => { selectNote(noteElement); }; function moveNote(event, deltaX, deltaY) { event.preventDefault(); const { clientX, clientY } = getPosition(event); noteElement.classList.add(NOTE_MOVING_CLASS); if (editable) { if (noteElement.classList.contains(NOTE_ANCHORED_CLASS)) { deleteNoteRef(containerElement, noteId); anchorElement = getTarget(clientX, clientY) || document.documentElement; addNoteRef(anchorElement, noteId); } else { anchorElement = document.documentElement; } } document.documentElement.insertBefore(containerElement, maskPageElement.getRootNode().host); noteElement.style.setProperty("left", (clientX - deltaX) + "px"); noteElement.style.setProperty("top", (clientY - deltaY) + "px"); noteElement.style.setProperty("position", "fixed"); displayMaskNote(); } function displayMaskNote() { if (anchorElement == document.documentElement || anchorElement == document.documentElement) { maskNoteElement.classList.remove(NOTE_MASK_MOVING_CLASS); } else { const boundingRectAnchor = anchorElement.getBoundingClientRect(); maskNoteElement.classList.add(NOTE_MASK_MOVING_CLASS); maskNoteElement.style.setProperty("top", boundingRectAnchor.y + "px"); maskNoteElement.style.setProperty("left", boundingRectAnchor.x + "px"); maskNoteElement.style.setProperty("width", boundingRectAnchor.width + "px"); maskNoteElement.style.setProperty("height", boundingRectAnchor.height + "px"); } } function selectNote(noteElement) { if (selectedNote) { selectedNote.classList.remove(NOTE_SELECTED_CLASS); } noteElement.classList.add(NOTE_SELECTED_CLASS); selectedNote = noteElement; } function getTarget(clientX, clientY) { const targets = Array.from(document.elementsFromPoint(clientX, clientY)).filter(element => element.tagName.toLowerCase() != NOTE_TAGNAME && !element.classList.contains(MASK_CLASS)); if (!targets.includes(document.documentElement)) { targets.push(document.documentElement); } let newTarget, target = targets[0], boundingRect = target.getBoundingClientRect(); newTarget = determineTargetElement("floor", target, clientX - boundingRect.left, getMatchedParents(target, "left")); if (newTarget == target) { newTarget = determineTargetElement("ceil", target, boundingRect.left + boundingRect.width - clientX, getMatchedParents(target, "right")); } if (newTarget == target) { newTarget = determineTargetElement("floor", target, clientY - boundingRect.top, getMatchedParents(target, "top")); } if (newTarget == target) { newTarget = determineTargetElement("ceil", target, boundingRect.top + boundingRect.height - clientY, getMatchedParents(target, "bottom")); } target = newTarget; while (boundingRect = target && target.getBoundingClientRect(), boundingRect && boundingRect.width <= SELECT_PX_THRESHOLD && boundingRect.height <= SELECT_PX_THRESHOLD) { target = target.parentElement; } return target; } function getMatchedParents(target, property) { let element = target, matchedParent, parents = []; do { const boundingRect = element.getBoundingClientRect(); if (element.parentElement && !element.parentElement.tagName.toLowerCase() != NOTE_TAGNAME && !element.classList.contains(MASK_CLASS)) { const parentBoundingRect = element.parentElement.getBoundingClientRect(); matchedParent = Math.abs(parentBoundingRect[property] - boundingRect[property]) <= SELECT_PX_THRESHOLD; if (matchedParent) { if (element.parentElement.clientWidth > SELECT_PX_THRESHOLD && element.parentElement.clientHeight > SELECT_PX_THRESHOLD && ((element.parentElement.clientWidth - element.clientWidth > SELECT_PX_THRESHOLD) || (element.parentElement.clientHeight - element.clientHeight > SELECT_PX_THRESHOLD))) { parents.push(element.parentElement); } element = element.parentElement; } } else { matchedParent = false; } } while (matchedParent && element); return parents; } function determineTargetElement(roundingMethod, target, widthDistance, parents) { if (Math[roundingMethod](widthDistance / SELECT_PX_THRESHOLD) <= parents.length) { target = parents[parents.length - Math[roundingMethod](widthDistance / SELECT_PX_THRESHOLD) - 1]; } return target; } }; const anchorNote = function anchorNote(event, noteElement, deltaX, deltaY) { event.preventDefault(); const { clientX, clientY } = getPosition(event); document.documentElement.style.removeProperty("user-select"); noteElement.classList.remove(NOTE_MOVING_CLASS); maskNoteElement.classList.remove(NOTE_MASK_MOVING_CLASS); maskPageElement.classList.remove(PAGE_MASK_ACTIVE_CLASS); const headerElement = noteElement.querySelector("header"); headerElement.ontouchmove = document.documentElement.onmousemove = null; let currentElement = anchorElement; let positionedElement; while (currentElement.parentElement && !positionedElement) { if (!FORBIDDEN_TAG_NAMES.includes(currentElement.tagName.toLowerCase())) { const currentElementStyle = getComputedStyle(currentElement); if (currentElementStyle.position != "static") { positionedElement = currentElement; } } currentElement = currentElement.parentElement; } if (!positionedElement) { positionedElement = document.documentElement; } const containerElement = noteElement.getRootNode().host; if (positionedElement == document.documentElement) { const firstMaskElement = document.querySelector("." + MASK_CLASS); document.documentElement.insertBefore(containerElement, firstMaskElement); } else { positionedElement.appendChild(containerElement); } const boundingRectPositionedElement = positionedElement.getBoundingClientRect(); const stylePositionedElement = window.getComputedStyle(positionedElement); const borderX = parseInt(stylePositionedElement.getPropertyValue("border-left-width")); const borderY = parseInt(stylePositionedElement.getPropertyValue("border-top-width")); noteElement.style.setProperty("position", "absolute"); noteElement.style.setProperty("left", (clientX - boundingRectPositionedElement.x - deltaX - borderX) + "px"); noteElement.style.setProperty("top", (clientY - boundingRectPositionedElement.y - deltaY - borderY) + "px"); }; const getPosition = function getPosition(event) { if (event.touches && event.touches.length) { const touch = event.touches[0]; return touch; } else { return event; } }; const onMouseUp = function onMouseUp(event) { if (highlightSelectionMode) { highlightSelection(); } if (removeHighlightMode) { let element = event.target, done; while (element && !done) { if (element.classList.contains(HIGHLIGHT_CLASS)) { document.querySelectorAll("." + HIGHLIGHT_CLASS + "[data-singlefile-highlight-id=" + JSON.stringify(element.dataset.singlefileHighlightId) + "]").forEach(highlightedElement => { resetHighlightedElement(highlightedElement); }); done = true; } element = element.parentElement; } } if (resizingNoteMode) { resizingNoteMode = false; document.documentElement.style.removeProperty("user-select"); maskPageElement.classList.remove(PAGE_MASK_ACTIVE_CLASS); document.documentElement.ontouchmove = document.documentElement.onmousemove = null; } if (movingNoteMode) { anchorNote(movingNoteMode.event || event, selectedNote, movingNoteMode.deltaX, movingNoteMode.deltaY); movingNoteMode = null; document.documentElement.ontouchmove = document.documentElement.onmousemove = null; } if (collapseNoteTimeout) { clearTimeout(collapseNoteTimeout); collapseNoteTimeout = null; } if (cuttingMode) { let element = event.target; if (document.documentElement != element && element.tagName.toLowerCase() != NOTE_TAGNAME) { element.classList.add(REMOVED_CONTENT_CLASS); removedElements.push(element); } } }; const maskNoteElement = getMaskElement("note-mask"); const maskPageElement = getMaskElement("page-mask", "single-file-page-mask"); let selectedNote, highlightSelectionMode, removeHighlightMode, resizingNoteMode, movingNoteMode, collapseNoteTimeout, cuttingMode; window.onresize = reflowNotes; document.documentElement.onmouseup = document.documentElement.ontouchend = onMouseUp; window.addEventListener("DOMContentLoaded", () => { processNode(document); reflowNotes(); document.querySelectorAll("single-file-note").forEach(noteElement => attachNoteListeners(noteElement)); }); })()</script><style>.single-file-highlight-yellow { background-color: #ffff7c !important; color: black !important; } span.single-file-highlight-yellow { display: inline !important; } .single-file-highlight-pink { background-color: #ffbbb6 !important; color: black !important; } span.single-file-highlight-pink { display: inline !important; } .single-file-highlight-blue { background-color: #95d0ff !important; color: black !important; } span.single-file-highlight-blue { display: inline !important; } .single-file-highlight-green { background-color: #93ef8d !important; color: black !important; } span.single-file-highlight-green { display: inline !important; } .single-file-highlight-hidden { background-color: inherit !important; color: inherit !important; } .single-file-mask { all: initial; display: contents !important; } .single-file-mask.single-file-page-mask { opacity: .99 !important; } single-file-note { all: initial !important; display: contents !important; } .single-file-hover { outline: 1px solid red !important; } .single-file-removed { display: none !important; float: none !important; position: static !important; visibility: collapse !important; }</style>